# -*- coding: utf-8 -*-
# encoding:utf-8

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

import redis

red_queue = "the_test_the_url_queue"
red_crawled_set = "the_test_url_has_crawled"

# connect to redis server
try:
	red = redis.Redis(host='localhost', port=6379, db=1)
	red2 = redis.Redis(host='localhost', port=6379, db=2)
except Exception,e:
	print str(e)



def re_crawl_url(url):
    red.lpush(red_queue, url)


def check_url(url):
	try:
	    if red.sadd(red_crawled_set, url):
	    	# print url,"ok"
	        red.lpush(red_queue, url)
	except Exception,e:
		print str(e)
